{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['../data_base/knowledge_db\\\\prompt_engineering\\\\1. 简介 Introduction.md', '../data_base/knowledge_db\\\\pumkin_book\\\\pumpkin_book.pdf']\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from dotenv import load_dotenv, find_dotenv\n",
    "\n",
    "# 读取本地/项目的环境变量。\n",
    "# find_dotenv()寻找并定位.env文件的路径\n",
    "# load_dotenv()读取该.env文件，并将其中的环境变量加载到当前的运行环境中  \n",
    "# 如果你设置的是全局的环境变量，这行代码则没有任何作用。\n",
    "_ = load_dotenv(find_dotenv())\n",
    "\n",
    "# 如果你需要通过代理端口访问，你需要如下配置\n",
    "# os.environ['HTTPS_PROXY'] = 'http://127.0.0.1:7890'\n",
    "# os.environ[\"HTTP_PROXY\"] = 'http://127.0.0.1:7890'\n",
    "\n",
    "# 获取folder_path下所有文件路径，储存在file_paths里\n",
    "file_paths = []\n",
    "folder_path = '../data_base/knowledge_db'\n",
    "for root, dirs, files in os.walk(folder_path):\n",
    "    for file in files:\n",
    "        file_path = os.path.join(root, file)\n",
    "        file_paths.append(file_path)\n",
    "print(file_paths[:3])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.document_loaders.pdf import PyMuPDFLoader\n",
    "from langchain.document_loaders.markdown import UnstructuredMarkdownLoader\n",
    "\n",
    "# 遍历文件路径并把实例化的loader存放在loaders里\n",
    "loaders = []\n",
    "\n",
    "for file_path in file_paths:\n",
    "\n",
    "    file_type = file_path.split('.')[-1]\n",
    "    if file_type == 'pdf':\n",
    "        loaders.append(PyMuPDFLoader(file_path))\n",
    "    elif file_type == 'md':\n",
    "        loaders.append(UnstructuredMarkdownLoader(file_path))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "f:\\Miniconda3\\envs\\jupyter38\\lib\\site-packages\\scipy\\__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.4\n",
      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n"
     ]
    }
   ],
   "source": [
    "# 下载文件并存储到text\n",
    "texts = []\n",
    "\n",
    "for loader in loaders: texts.extend(loader.load())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "载入后的变量类型为langchain_core.documents.base.Document, 文档变量类型同样包含两个属性\n",
    "\n",
    "- page_content 包含该文档的内容。\n",
    "- meta_data 为文档相关的描述性数据。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "每一个元素的类型：<class 'langchain_core.documents.base.Document'>.\n",
      "------\n",
      "该文档的描述性数据：{'source': '../data_base/knowledge_db\\\\pumkin_book\\\\pumpkin_book.pdf', 'file_path': '../data_base/knowledge_db\\\\pumkin_book\\\\pumpkin_book.pdf', 'page': 0, 'total_pages': 196, 'format': 'PDF 1.5', 'title': '', 'author': '', 'subject': '', 'keywords': '', 'creator': 'LaTeX with hyperref', 'producer': 'xdvipdfmx (20200315)', 'creationDate': \"D:20231117152045-00'00'\", 'modDate': '', 'trapped': ''}\n",
      "------\n",
      "查看该文档的内容:\n",
      "\u0001本\u0003:2.0.0\n",
      "发布日期:2023.11\n",
      "南  ⽠  书\n",
      "PUMPKIN\n",
      "B  O  O  K\n",
      "谢\t睿 \u000b州 贾彬彬\n",
      "\n"
     ]
    }
   ],
   "source": [
    "text = texts[1]\n",
    "print(f\"每一个元素的类型：{type(text)}.\", \n",
    "    f\"该文档的描述性数据：{text.metadata}\", \n",
    "    f\"查看该文档的内容:\\n{text.page_content[0:]}\", \n",
    "    sep=\"\\n------\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "# 切分文档\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=500, chunk_overlap=50)\n",
    "\n",
    "split_docs = text_splitter.split_documents(texts)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 二、构建Chroma向量库"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[WARNING][2024-12-04 19:53:45.664] redis_rate_limiter.py:21 [t:20200]: No redis installed, RedisRateLimiter unavailable. Ignore this warning if you don't need to use qianfan SDK in distribution environment\n"
     ]
    }
   ],
   "source": [
    "# 使用 OpenAI Embedding\n",
    "# from langchain.embeddings.openai import OpenAIEmbeddings\n",
    "# 使用百度千帆 Embedding\n",
    "from langchain.embeddings.baidu_qianfan_endpoint import QianfanEmbeddingsEndpoint\n",
    "# 使用我们自己封装的智谱 Embedding，需要将封装代码下载到本地使用\n",
    "# from zhipuai_embedding import ZhipuAIEmbeddings\n",
    "\n",
    "# 定义 Embeddings\n",
    "# embedding = OpenAIEmbeddings() \n",
    "# embedding = ZhipuAIEmbeddings()\n",
    "embedding = QianfanEmbeddingsEndpoint()\n",
    "\n",
    "# 定义持久化路径\n",
    "persist_directory = '../data_base/vector_db/chroma'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf '../data_base/vector_db/chroma'  # 删除旧的数据库文件（如果文件夹中有文件的话），windows电脑请手动删除\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.vectorstores.chroma import Chroma\n",
    "\n",
    "vectordb = Chroma.from_documents(\n",
    "    documents=split_docs[:5], # 为了速度，只选择前 20 个切分的 doc 进行生成；使用千帆时因QPS限制，建议选择前 5 个doc\n",
    "    embedding=embedding,\n",
    "    persist_directory=persist_directory  # 允许我们将persist_directory目录保存到磁盘上\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "确保通过运行 vectordb.persist 来持久化向量数据库，以便我们在未来的课程中使用。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "向量库中存储的数量：25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\AppData\\Local\\Temp\\ipykernel_17360\\1292894578.py:1: LangChainDeprecationWarning: Since Chroma 0.4.x the manual persistence method is no longer supported as docs are automatically persisted.\n",
      "  vectordb.persist()\n"
     ]
    }
   ],
   "source": [
    "vectordb.persist()\n",
    "print(f\"向量库中存储的数量：{vectordb._collection.count()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 三、向量检索\n",
    "##### 3.1 相似度检索\n",
    "Chroma的相似度搜索使用的是余弦距离，即：\n",
    "\n",
    "当你需要数据库返回严谨的按余弦相似度排序的结果时可以使用similarity_search函数。\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "检索到的内容数：3\n"
     ]
    }
   ],
   "source": [
    "question=\"什么是大语言模型\"\n",
    "sim_docs = vectordb.similarity_search(question,k=3)\n",
    "print(f\"检索到的内容数：{len(sim_docs)}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "检索到的第0个内容: \n",
      "网络上有许多关于提示词（Prompt， 本教程中将保留该术语）设计的材料，例如《30 prompts everyone has to know》之类的文章，这些文章主要集中在 ChatGPT 的 Web 界面上，许多人在使用它执行特定的、通常是一次性的任务。但我们认为，对于开发人员，大语言模型（LLM） 的更强大功能是能通过 API 接口调用，从而快速构建软件应用程序。实际上，我们了解到 Deep\n",
      "--------------\n",
      "检索到的第1个内容: \n",
      "网络上有许多关于提示词（Prompt， 本教程中将保留该术语）设计的材料，例如《30 prompts everyone has to know》之类的文章，这些文章主要集中在 ChatGPT 的 Web 界面上，许多人在使用它执行特定的、通常是一次性的任务。但我们认为，对于开发人员，大语言模型（LLM） 的更强大功能是能通过 API 接口调用，从而快速构建软件应用程序。实际上，我们了解到 Deep\n",
      "--------------\n",
      "检索到的第2个内容: \n",
      "与基础语言模型不同，指令微调 LLM 通过专门的训练，可以更好地理解并遵循指令。举个例子，当询问“法国的首都是什么？”时，这类模型很可能直接回答“法国的首都是巴黎”。指令微调 LLM 的训练通常基于预训练语言模型，先在大规模文本数据上进行预训练，掌握语言的基本规律。在此基础上进行进一步的训练与微调（finetune），输入是指令，输出是对这些指令的正确回复。有时还会采用RLHF（reinforce\n",
      "--------------\n"
     ]
    }
   ],
   "source": [
    "for i, sim_doc in enumerate(sim_docs):\n",
    "    print(f\"检索到的第{i}个内容: \\n{sim_doc.page_content[:200]}\", end=\"\\n--------------\\n\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 3.2 MMR检索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MMR 检索到的第0个内容: \n",
      "网络上有许多关于提示词（Prompt， 本教程中将保留该术语）设计的材料，例如《30 prompts everyone has to know》之类的文章，这些文章主要集中在 ChatGPT 的 Web 界面上，许多人在使用它执行特定的、通常是一次性的任务。但我们认为，对于开发人员，大语言模型（LLM） 的更强大功能是能通过 API 接口调用，从而快速构建软件应用程序。实际上，我们了解到 Deep\n",
      "--------------\n",
      "MMR 检索到的第1个内容: \n",
      "与基础语言模型不同，指令微调 LLM 通过专门的训练，可以更好地理解并遵循指令。举个例子，当询问“法国的首都是什么？”时，这类模型很可能直接回答“法国的首都是巴黎”。指令微调 LLM 的训练通常基于预训练语言模型，先在大规模文本数据上进行预训练，掌握语言的基本规律。在此基础上进行进一步的训练与微调（finetune），输入是指令，输出是对这些指令的正确回复。有时还会采用RLHF（reinforce\n",
      "--------------\n",
      "MMR 检索到的第2个内容: \n",
      "前言\n",
      "“周志华老师的《机器学习》（西瓜书）是机器学习领域的经典入门教材之一，周老师为了使尽可能多的读\n",
      "者通过西瓜书对机器学习有所了解, 所以在书中对部分公式的推导细节没有详述，但是这对那些想深究公式推\n",
      "导细节的读者来说可能“不太友好”，本书旨在对西瓜书里比较难理解的公式加以解析，以及对部分公式补充\n",
      "具体的推导细节。”\n",
      "读到这里，大家可能会疑问为啥前面这段话加了引号，因为这只是我们最初的遐想，后来我\n",
      "--------------\n"
     ]
    }
   ],
   "source": [
    "mmr_docs = vectordb.max_marginal_relevance_search(question,k=3)\n",
    "for i, sim_doc in enumerate(mmr_docs):\n",
    "    print(f\"MMR 检索到的第{i}个内容: \\n{sim_doc.page_content[:200]}\", end=\"\\n--------------\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "jupyter38",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
